#===============================================================================
# 8_text_study_figure_11_appH.R
# Purpose: To replicate Figure 18 of the paper: 
#          "IRR between participants and researcher gold standard"
#===============================================================================

# PACKAGES
#===============================================================================
library(dplyr)
library(tidyr)
library(rio)
library(stringr)
library(irr)
library(ggplot2)
## dutchmasters gives prettier colors used in appendix, 
## but is not available for R 4.4.2
#library(dutchmasters)


# DATA
#===============================================================================
# - load dataset with all data annotations form all participants and all 3 rounds
ra_db <- read.csv("./data/text-experiment-merged-dataset.csv")

# - load our gold standard
gold_db <- read.csv("./data/SESSION_01_data_GOLD.csv")


# MAIN
#===============================================================================

#-------------------------------------------------------------------------------
# [ A ] IRR between each RA and gold standard
#-------------------------------------------------------------------------------

# - calculate IRR between the gold standard and each RA and for the Basic and
#.  Advanced coding rounds

# - initialize empty dataset to fill out
out_db <- NULL

# - unique list of participants; sessions; and variables
participants <- unique(ra_db$PID)
sessions <- c(1, 3)
# ... not including the emotion variables here
variables <- c("DIRECTED_AT", "NEG_TONE", "CONS_VIEW", "PROGR_VIEW",
               "GENDER")

# - iterate through participants, sessions, and variables; and calculate IRR 
#.  compared to our Gold Standard
for (part in participants) {
  # - keep track of key sociodemographic variables 
  # ... gender
  part_gender <- ra_db$GENDERCODER[ra_db$PID == part][1]
  # ... ideology
  part_ideocont <- ra_db$IDEO[ra_db$PID == part][1]
  part_cons <- ifelse(as.numeric(part_ideocont) >= 5, "Conservative", 
                      "Progressive")
  
  # - iterate through the 2 annotation sessions: Basic & Advanced
  for (session in sessions) {
    # ... training session
    training_round <- ifelse(session == 1, "Basic", "Advanced")
    
    # - pull data of interest
    iter_db <- ra_db %>%
      filter(SESSION == session,
             PID == part,
             # - ONLY USE THE FIRST 150 MESSAGES
             as.numeric(ID_ADHOC) < 151)
    
    # - iterate through annotations tasks
    for (variable in variables) {
      
      ra_data <- iter_db[,c("ID_ADHOC", variable)]
      colnames(ra_data)[2] <- "ra"
      gold_data <- gold_db[,c("ID_ADHOC", variable)]
      colnames(gold_data)[2] <- "gold"
      
      # - only continue if we have data for this participant and round
      if (nrow(ra_data) > 0) {
        to_test <- as.matrix(left_join(ra_data, gold_data) %>%
                               dplyr::select(-ID_ADHOC))
        irr_cohen <- kappa2(to_test)$value
        irr_krip <- kripp.alpha(to_test)$value
        # - save this data in the output dataset
        new_row <- data.frame(
          PID = part,
          gender = part_gender,
          ideology = part_cons,
          training = training_round,
          task = variable,
          irr_cohen = irr_cohen,
          irr_krip = irr_krip
        )
        out_db <- rbind(out_db, new_row)
      }
    }
  }
}


#-------------------------------------------------------------------------------
# [ B ] Analysis
#-------------------------------------------------------------------------------
# - average IRR across the 5 tasks, by RA, and by training session
plot_db01 <- out_db %>%
  group_by(PID, training) %>%
  summarise(gender = gender[1],
            ideology = ideology[1],
            pe = mean(irr_cohen)) %>%
  mutate(task = "ALL")

plot_db02 <- out_db %>%
  dplyr::select(-irr_krip) %>%
  mutate(task = recode(task,
                       `DIRECTED_AT` = "Directed at",
                       `NEG_TONE` = "Negative\ntone",
                       `CONS_VIEW` = "Conservative\nview",
                       `PROGR_VIEW` = "Progressive\nview",
                       `GENDER` = "Gender\nissue")) %>%
  rename(pe = irr_cohen)

# ... only keep observations for those that completed the Basic and Advanced
#     training
both_sessions <- names(which(table(plot_db02$PID) == 10))

plot_db <- rbind(plot_db01, plot_db02) %>%
  as.data.frame() %>%
  filter(PID %in% both_sessions) %>%
  mutate(Task = factor(task, levels = c(
    "ALL",
    "Directed at",
    "Negative\ntone",
    "Conservative\nview",
    "Progressive\nview",
    "Gender\nissue"
  )),
  training = factor(training, levels = rev(c(
    "Basic", "Advanced"
  )))) %>%
  arrange(task, training, ideology, pe) %>%
  mutate(PID = factor(PID, levels = unique(PID)),
         main_task = ifelse(task == "ALL", 1, 0),
         training = factor(training, levels = c(
           "Basic", "Advanced"
         )))

ggplot(plot_db,
       aes(x = PID, y = pe)) +
  geom_polygon(inherit.aes = FALSE, 
               data = data.frame(x = c(0, 0, 7.5, 7.5), y = c(0, 1, 1, 0)),
               aes(x = x , y = y),
               fill = "gray90", alpha = 0.5) +
  geom_point(aes(color = Task, size = as.character(main_task)),
             alpha = 0.8) +
  geom_point(aes(size = as.character(main_task)), pch = 1) +
  geom_text(inherit.aes = FALSE,
            data = data.frame(x = 3.8, y = 0.95, label = "Conservative coders"),
            aes(x = x, y = y, label = label, angle = 90))+
  geom_text(inherit.aes = FALSE,
            data = data.frame(x = 16, y = 0.95, label = "Progressive coders"),
            aes(x = x, y = y, label = label, angle = 90))+
  geom_hline(yintercept = 0, color = "black")  +
  geom_hline(yintercept = 0.5, color = "black") +
  facet_wrap(~ training) +
  coord_flip() +
  scale_x_discrete("Participant ID") +
  scale_y_continuous("Cohen's Kappa (v. Researcher's Gold Standard)") +
 # uncomment below for prettier colors
 # scale_colour_dutchmasters("little_street") +
  scale_size_manual("", values = c(4, 7), guide = "none") +
  guides(colour = guide_legend(override.aes = list(size=6))) +
  theme(strip.text = element_text(size = 16),
        panel.background = element_blank(),
        panel.grid.major = element_line(color = "gray90"),
        axis.text = element_text(size = 12))

ggsave("./results/appendix_h/appH_11_irr_gold_standard.pdf",
       width = 30, height = 18, units = "cm", dpi = 300)
